CycleGAN is an unpaired image-to-image translation network using cycle-consistent generative adversarial networks.
DOWNLOAD_MODELS = 1
Training the network using the data set ukiyoe2photo produces the transform from Ukiyo-e style paintings to photos and its inverse transform, photos to Ukiyo-e style paintings.
Training the network using the data set handB_backP produces the transform from backpack images to handbag images and its inverse transform, handbag images to backpack images.
Training the network using the data set manwoman produces the transform from images of men to images of women and its inverse transform.
Training the network using the data set monet2photo produces the transform from Monet's paintings to photos and its inverse transform, photos to Monet-styled paintings.
Training the network using the data set vangogh2photo produces the transform from Van Gogh's paintings to photos and its inverse transform, photos to Van Gogh-styled paintings.
Training the network using the data set iphone2dslr_flower produces a transform from normal photos to photos with shallow depth of field and its inverse transform.
Examination of Loss Function Components
This experiment examines the effect of each component of the loss function on the resulting image.
from __future__ import print_function
import os.path
from torchvision import transforms
from Dataset import ImageSet
import torch
from torch.utils.data import DataLoader
import ModelTestFuncs as MTF
import matplotlib.pyplot as plt
import numpy as np
import imageio
from torch.autograd import Variable
from torchvision import utils as tv
import os
reload(MTF)
imageScale = 1;
images = ImageSet();
path = './img/test_images/'
test_transforms = [transforms.ToTensor(),
transforms.Normalize((0.5,0.5,0.5), (0.5,0.5,0.5))];
images.loadImageSet(path, test_transforms, "test", imageScale);
imgLoader = DataLoader(images, 1, shuffle=False);
imgs = [];
for i in range(1,7):
imgs.append(imageio.imread("./img/test_images/IMG_"+str(i)+".JPG"));
Disables autoscrolling in result windows.
%%javascript
IPython.OutputArea.prototype._should_scroll = function(lines) {
return false;
}
model = 'ukiyoe2photo';
modelfile = torch.load("./model/"+model+"/F.data");
F = MTF.prepModel(modelfile);
for i, img in enumerate(imgLoader):
imgsize = img['img'].size();
img_gpu = torch.cuda.FloatTensor(1,3,imgsize[2],imgsize[3]);
img_var = Variable(img_gpu.copy_(img['img']));
result = 0.5*(F(img_var).data+1.0);
result = result.cpu().numpy().squeeze().transpose(1,2,0);
plt.figure(figsize = (16,12))
plt.subplot(1,2,1);
plt.imshow(imgs[i])
plt.axis('off');
plt.subplot(1,2,2);
plt.imshow(result)
plt.axis('off');
plt.show();
del img_gpu
del F
Back to contents
This model converts images of handbacks to backpacks and also performs the inverse transform, backpacks to handbags.
from PIL import Image
imageScale = 1;
im_size = 128
images = ImageSet();
path = './img/test_images_bag/handbag/'
test_transforms = [transforms.Resize(int(im_size*1.12), Image.BICUBIC),
transforms.CenterCrop(im_size),
transforms.ToTensor(),
transforms.Normalize((0.5,0.5,0.5),(0.5,0.5,0.5))];
images.loadImageSet(path, test_transforms, "test", imageScale);
imgLoader = DataLoader(images, 1, shuffle=False);
model = 'handB_backP';
modelfile = torch.load("./model/"+model+"/G.data");
F = MTF.prepModel(modelfile);
for i, img in enumerate(imgLoader):
imgsize = img['img'].size();
img_gpu = torch.cuda.FloatTensor(1,3,imgsize[2],imgsize[3]);
img_var = Variable(img_gpu.copy_(img['img']));
result = 0.5*(F(img_var).data+1.0);
result = result.cpu().numpy().squeeze().transpose(1,2,0);
plt.figure(figsize = (10,6))
plt.subplot(1,2,1);
plt.imshow((0.5*(img['img'].numpy()+1)).squeeze().transpose(1,2,0))
plt.axis('off');
plt.subplot(1,2,2);
plt.imshow(result)
plt.axis('off');
plt.show();
if (DOWNLOAD_MODELS):
MTF.downloadModel("handB_backP", "F");
imageScale = 1;
images = ImageSet();
path = './img/test_images_bag/backpack/'
test_transforms = [transforms.Resize(int(im_size*1.12), Image.BICUBIC),
transforms.CenterCrop(im_size),
transforms.ToTensor(),
transforms.Normalize((0.5,0.5,0.5),(0.5,0.5,0.5))];
images.loadImageSet(path, test_transforms, "test", imageScale);
imgLoader = DataLoader(images, 1, shuffle=False);
modelfile = torch.load("./modelDL/"+model+"/F.data");
F = MTF.prepModel(modelfile);
for i, img in enumerate(imgLoader):
imgsize = img['img'].size();
img_gpu = torch.cuda.FloatTensor(1,3,imgsize[2],imgsize[3]);
img_var = Variable(img_gpu.copy_(img['img']));
result = 0.5*(F(img_var).data+1.0);
result = result.cpu().numpy().squeeze().transpose(1,2,0);
plt.figure(figsize = (10,6))
plt.subplot(1,2,1);
plt.imshow((0.5*(img['img'].numpy()+1)).squeeze().transpose(1,2,0))
plt.axis('off');
plt.subplot(1,2,2);
plt.imshow(result)
plt.axis('off');
plt.show();
del img_gpu
del F
os.remove("./modelDL/"+model+"/F.data");
else:
for i in range(1,6):
plt.figure(figsize = (10,6))
plt.imshow(imageio.imread("./outputs/test_bags/result"+str(i)+".png"));
plt.axis('off');
Back to contents
This model converts images of men to images of women and performs the inverse transform, women to men.
if (DOWNLOAD_MODELS):
MTF.downloadModel("manwoman", "G");
imageScale = 1;
images = ImageSet();
path = './img/test_images_face/men/'
test_transforms = [transforms.ToTensor(),
transforms.Normalize((0.5,0.5,0.5), (0.5,0.5,0.5))];
images.loadImageSet(path, test_transforms, "test", imageScale);
imgLoader = DataLoader(images, 1, shuffle=False);
model = 'manwoman';
modelfile = torch.load("./modelDL/"+model+"/G.data");
F = MTF.prepModel(modelfile);
for i, img in enumerate(imgLoader):
imgsize = img['img'].size();
img_gpu = torch.cuda.FloatTensor(1,3,imgsize[2],imgsize[3]);
img_var = Variable(img_gpu.copy_(img['img']));
result = 0.5*(F(img_var).data+1.0);
result = result.cpu().numpy().squeeze().transpose(1,2,0);
plt.figure(figsize = (14,10))
plt.subplot(1,2,1);
plt.imshow((0.5*(img['img'].numpy()+1)).squeeze().transpose(1,2,0))
plt.axis('off');
plt.subplot(1,2,2);
plt.imshow(result)
plt.axis('off');
plt.show();
MTF.downloadModel("manwoman", "F");
imageScale = 1;
images = ImageSet();
path = './img/test_images_face/women/'
test_transforms = [transforms.ToTensor(),
transforms.Normalize((0.5,0.5,0.5), (0.5,0.5,0.5))];
images.loadImageSet(path, test_transforms, "test", imageScale);
imgLoader = DataLoader(images, 1, shuffle=False);
modelfile = torch.load("./modelDL/"+model+"/F.data");
F = MTF.prepModel(modelfile);
for i, img in enumerate(imgLoader):
imgsize = img['img'].size();
img_gpu = torch.cuda.FloatTensor(1,3,imgsize[2],imgsize[3]);
img_var = Variable(img_gpu.copy_(img['img']));
result = 0.5*(F(img_var).data+1.0);
result = result.cpu().numpy().squeeze().transpose(1,2,0);
plt.figure(figsize = (14,10))
plt.subplot(1,2,1);
plt.imshow((0.5*(img['img'].numpy()+1)).squeeze().transpose(1,2,0))
plt.axis('off');
plt.subplot(1,2,2);
plt.imshow(result)
plt.axis('off');
plt.show();
del img_gpu
del F
os.remove("./modelDL/manwoman/G.data");
os.remove("./modelDL/manwoman/F.data");
else:
for i in range(1,4):
plt.figure(figsize = (14,10))
plt.subplot(1,2,1);
imgi = imageio.imread("./img/test_images_face/men/face_"+str(i)+".jpg");
plt.imshow(imgi)
plt.axis('off');
plt.subplot(1,2,2);
imgo = imageio.imread("./outputs/manwoman/men/out_face_"+str(i)+".jpg");
plt.imshow(imgo)
plt.axis('off');
for i in range(1,4):
plt.figure(figsize = (14,10))
plt.subplot(1,2,1);
imgi = imageio.imread("./img/test_images_face/women/face_"+str(i)+".jpg");
plt.imshow(imgi)
plt.axis('off');
plt.subplot(1,2,2);
imgo = imageio.imread("./outputs/manwoman/women/out_face_"+str(i)+".jpg");
plt.imshow(imgo)
plt.axis('off');
imageScale = 1;
images = ImageSet();
path = './img/test_images/'
test_transforms = [transforms.ToTensor(),
transforms.Normalize((0.5,0.5,0.5), (0.5,0.5,0.5))];
images.loadImageSet(path, test_transforms, "test", imageScale);
imgLoader = DataLoader(images, 1, shuffle=False);
imgs = [];
for i in range(1,7):
imgs.append(imageio.imread("./img/test_images/IMG_"+str(i)+".JPG"));
model = 'monet2photo'
if (DOWNLOAD_MODELS):
direct = "F";
MTF.downloadModel(model, direct);
modelfile = torch.load("./modelDL/"+model+"/" + direct + ".data");
F = MTF.prepModel(modelfile);
for i, img in enumerate(imgLoader):
imgsize = img['img'].size();
img_gpu = torch.cuda.FloatTensor(1,3,imgsize[2],imgsize[3]);
img_var = Variable(img_gpu.copy_(img['img']));
result = 0.5*(F(img_var).data+1.0);
result = result.cpu().numpy().squeeze().transpose(1,2,0);
plt.figure(figsize = (16,12))
plt.subplot(1,2,1);
plt.imshow(imgs[i])
plt.axis('off');
plt.subplot(1,2,2);
plt.imshow(result)
plt.axis('off');
plt.show();
del img_gpu
del F
os.remove("./modelDL/"+model+"/" + direct + ".data");
else:
for i in range(1,7):
plt.figure(figsize = (16,12))
plt.subplot(1,2,1);
plt.imshow(imgs[i-1])
plt.axis('off');
plt.subplot(1,2,2);
imgo = imageio.imread("./outputs/"+model+"/test_images/out_IMG_"+str(i)+".JPG");
plt.imshow(imgo)
plt.axis('off');
plt.show();
Back to contents
This model takes photos and renders them in the style of Van Gogh paintings.
model = 'vangogh2photo'
if (DOWNLOAD_MODELS):
direct = "F";
MTF.downloadModel(model, direct);
modelfile = torch.load("./modelDL/"+model+"/" + direct + ".data");
F = MTF.prepModel(modelfile);
for i, img in enumerate(imgLoader):
imgsize = img['img'].size();
img_gpu = torch.cuda.FloatTensor(1,3,imgsize[2],imgsize[3]);
img_var = Variable(img_gpu.copy_(img['img']));
result = 0.5*(F(img_var).data+1.0);
result = result.cpu().numpy().squeeze().transpose(1,2,0);
plt.figure(figsize = (16,12))
plt.subplot(1,2,1);
plt.imshow(imgs[i])
plt.axis('off');
plt.subplot(1,2,2);
plt.imshow(result)
plt.axis('off');
plt.show();
del img_gpu
del F
os.remove("./modelDL/"+model+"/" + direct + ".data");
else:
for i in range(1,7):
plt.figure(figsize = (16,12))
plt.subplot(1,2,1);
plt.imshow(imgs[i-1])
plt.axis('off');
plt.subplot(1,2,2);
imgo = imageio.imread("./outputs/"+model+"/test_images/out_IMG_"+str(i)+".JPG");
plt.imshow(imgo)
plt.axis('off');
plt.show();
Back to contents
This model transforms photos of flowers taken without a shallow depth of field to photos of flowers with a shallow depth of field.
imageScale = 1;
images = ImageSet();
path = './img/test_flowers/'
test_transforms = [transforms.ToTensor(),
transforms.Normalize((0.5,0.5,0.5), (0.5,0.5,0.5))];
images.loadImageSet(path, test_transforms, "test", imageScale);
imgLoader = DataLoader(images, 1, shuffle=False);
model = 'iphone2dslr_flower'
if (DOWNLOAD_MODELS):
direct = "G";
MTF.downloadModel(model, direct);
modelfile = torch.load("./modelDL/"+model+"/" + direct + ".data");
F = MTF.prepModel(modelfile);
for i, img in enumerate(imgLoader):
imgsize = img['img'].size();
img_gpu = torch.cuda.FloatTensor(1,3,imgsize[2],imgsize[3]);
img_var = Variable(img_gpu.copy_(img['img']));
result = 0.5*(F(img_var).data+1.0);
result = result.cpu().numpy().squeeze().transpose(1,2,0);
plt.figure(figsize = (16,12))
plt.subplot(1,2,1);
plt.imshow((0.5*(img['img'].numpy()+1)).squeeze().transpose(1,2,0))
plt.axis('off');
plt.subplot(1,2,2);
plt.imshow(result)
plt.axis('off');
plt.show();
del img_gpu
del F
os.remove("./modelDL/"+model+"/" + direct + ".data");
else:
for i in range(1,7):
plt.figure(figsize = (16,12))
plt.subplot(1,2,1);
imgi = imageio.imread("./img/test_flowers/flower_"+str(i)+".jpg");
plt.imshow(imgi)
plt.axis('off');
plt.subplot(1,2,2);
imgo = imageio.imread("./outputs/test_flowers/out_flower_"+str(i)+".jpg");
plt.imshow(imgo)
plt.axis('off');
plt.show();
Back to contents
There are two GAN network losses, a cycle-consistency loss, and an identity loss.
plt.figure(figsize = (16,12))
plt.subplot(2,2,1);
img0 = imageio.imread("./img/test_images/IMG_1.JPG");
plt.imshow(img0)
plt.axis('off');
plt.title("Original Image")
plt.subplot(2,2,2);
img1 = imageio.imread("./outputs/ukiyoe2photo/test_images/out_IMG_1.JPG");
plt.imshow(img1)
plt.axis('off');
plt.title("Full Loss Function")
plt.subplot(2,2,3);
img2 = imageio.imread("./outputs/test_loss/out_IMG_1_gan.JPG");
plt.imshow(img2)
plt.axis('off');
plt.title("No Cycle-Consistency Loss")
plt.subplot(2,2,4);
img3 = imageio.imread("./outputs/test_loss/out_IMG_1_noident.JPG");
plt.imshow(img3)
plt.axis('off');
plt.title("No Identity Loss")
plt.show();